#!/usr/bin/env python
# -*- coding: utf-8 -*-

import codecs
import copy
import csv
import json
import logging
import logging.config
import math
import os
import random
import re
import sqlite3
import sys
import warnings
from collections import OrderedDict
from datetime import date, datetime, timedelta
from pathlib import Path
from time import sleep
import datetime
import requests
from lxml import etree
from requests.adapters import HTTPAdapter
from tqdm import tqdm

import const
from util import csvutil
from util.dateutil import convert_to_days_ago
from util.notify import push_deer

warnings.filterwarnings("ignore")

# 如果日志文件夹不存在，则创建
if not os.path.isdir("log/"):
    os.makedirs("log/")
logging_path = os.path.split(os.path.realpath(__file__))[0] + os.sep + "logging.conf"
logging.config.fileConfig(logging_path)
logger = logging.getLogger("twitter")


class Twitter(object):
    def __init__(self, config):
        """Weibo类初始化"""
        self.validate_config(config)
        self.filter = config["filter"]  # 取值范围为0、1,程序默认值为0,代表要爬取用户的全部微博,1代表只爬取用户的原创微博
        self.remove_html_tag = config[
            "remove_html_tag"
        ]  # 取值范围为0、1, 0代表不移除微博中的html tag, 1代表移除
        since_date = config["since_date"]
        if isinstance(since_date, int):
            since_date = date.today() - timedelta(since_date)
        since_date = str(since_date)
        self.since_date = since_date  # 起始时间，即爬取发布日期从该值到现在的微博，形式为yyyy-mm-dd
        self.start_page = config.get("start_page", 1)  # 开始爬的页，如果中途被限制而结束可以用此定义开始页码
        self.write_mode = config[
            "write_mode"
        ]  # 结果信息保存类型，为list形式，可包含csv、mongo和mysql三种类型
        self.original_pic_download = config[
            "original_pic_download"
        ]  # 取值范围为0、1, 0代表不下载原创微博图片,1代表下载


        self.headers = {
            "Content-Type": "application/json",
            "Accept": "*/*",
            "Authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
            "Accept-Language": "zh-cn",
            "Host": "twitter.com",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1 Safari/605.1.15",
            "Referer": "https://twitter.com/thurman_carlton",
            "Connection": "keep-alive",
            "Cookie": "_ga=GA1.2.1346708917.1661309244; _gid=GA1.2.1051927047.1661309244; gt=1562315620112207872; ct0=e4730a348248f600f1e8ce8444e89bd2; guest_id=v1%3A166130923308134689; guest_id_ads=v1%3A166130923308134689; guest_id_marketing=v1%3A166130923308134689; personalization_id=\"v1_HaQF4n8WdtK4HNh6YIKRwQ==\"",
            "x-guest-token": "1562315620112207872",
            "x-twitter-client-language": "zh-cn",
            "x-csrf-token": "e4730a348248f600f1e8ce8444e89bd2",
            "x-twitter-active-user": "yes"
        }
        # 不需要登入 headers
        self.headers_no_login = {
            "Content-Type": "application/json",
            "Accept": "*/*",
            "Authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
            "Accept-Language": "zh-cn",
            "Host": "twitter.com",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1 Safari/605.1.15",
            "Referer": "https://twitter.com/thurman_carlton",
            "Connection": "keep-alive",
            "Cookie": "_ga=GA1.2.1346708917.1661309244; _gid=GA1.2.1051927047.1661309244; gt=1562315620112207872; ct0=e4730a348248f600f1e8ce8444e89bd2; guest_id=v1%3A166130923308134689; guest_id_ads=v1%3A166130923308134689; guest_id_marketing=v1%3A166130923308134689; personalization_id=\"v1_HaQF4n8WdtK4HNh6YIKRwQ==\"",
            "x-guest-token": "1562315620112207872",
            "x-twitter-client-language": "zh-cn",
            "x-csrf-token": "e4730a348248f600f1e8ce8444e89bd2",
            "x-twitter-active-user": "yes"
        }
        self.headers_bk = {
            "Host": "twitter.com",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",
            "Accept": "*/*",
            "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",

            "content-type": "application/json",
            "x-twitter-auth-type": "OAuth2Session",
            "x-twitter-client-language": "en",
            "x-twitter-active-user": "no",
            "x-csrf-token": "8b4c317e9df2d7dc5d00abbaa17884560abc4726a21fc053c7f4eab77891bfcd6e8f10efb6f479c2f797e5f5e67a90537d5ba1b4c48fcfda449f4735bf11ca7ebb8fcdad10c4a5eb76c0b41a647ac046",
            "Sec-Fetch-Dest": "empty",
            "Sec-Fetch-Mode": "cors",
            "Sec-Fetch-Site": "same-origin",
            "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
            "Referer": "https://twitter.com/HongKong_Doll/media",
            "Connection": "keep-alive",
            "Pragma": "no-cache",
            "Cache-Control": "no-cache",
            "TE": "trailers",
            "Cookie": "guest_id_marketing=v1%3A164190143802032484; guest_id_ads=v1%3A164190143802032484; personalization_id='v1_5KGeFKS26tQLzUE52z3VeA=='; guest_id=v1%3A164190143802032484; _ga=GA1.2.806283036.1641901441; kdt=A7tzmJnrNqAkuatT4P9nDFmAa2VcN0cOUmO4EH4S; ct0=8b4c317e9df2d7dc5d00abbaa17884560abc4726a21fc053c7f4eab77891bfcd6e8f10efb6f479c2f797e5f5e67a90537d5ba1b4c48fcfda449f4735bf11ca7ebb8fcdad10c4a5eb76c0b41a647ac046; twid=u%3D1379233777403060226; auth_token=659dcccf7ac318e4c910eb9f5f3e98667523119c; _gid=GA1.2.1908623587.1660633991; lang=en",
        }
        self.down_headers_bk = {
            "Host": "pbs.twimg.com",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
            "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
            "Upgrade-Insecure-Requests": "1",
            "Connection": "keep-alive",
            "Sec-Fetch-Dest": "document",
            "Sec-Fetch-Mode": "navigate",
            "Sec-Fetch-Site": "none",
            "Sec-Fetch-User": "?1",
            "Range": "bytes=98304-",
            "If-Range": "Thu, 18 Aug 2022 12:02:26 GMT",
            "TE": "trailers",
        }
        self.down_headers = {

            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",

        }
        self.mysql_config = config.get("mysql_config")  # MySQL数据库连接配置，可以不填
        user_id_list = config["user_id_list"]
        query_list = config.get("query_list") or []
        if isinstance(query_list, str):
            query_list = query_list.split(",")
        self.query_list = query_list
        if not isinstance(user_id_list, list):
            if not os.path.isabs(user_id_list):
                user_id_list = (
                    os.path.split(os.path.realpath(__file__))[0] + os.sep + user_id_list
                )
            self.user_config_file_path = user_id_list  # 用户配置文件路径
            user_config_list = self.get_user_config_list(user_id_list)
        else:
            self.user_config_file_path = ""
            user_config_list = [
                {
                    "user_id": user_id,
                    "since_date": self.since_date,
                    "query_list": query_list,
                }
                for user_id in user_id_list
            ]
        self.user_config_list = user_config_list
        self.cursor = ""
        self.userid = ""

    def validate_config(self, config):
        """验证配置是否正确"""

        # 验证filter、original_pic_download、retweet_pic_download、original_video_download、retweet_video_download
        argument_list = [
            "filter",
            "original_pic_download",
            "retweet_pic_download",
            "original_video_download",
            "retweet_video_download",
            "download_comment",
            "download_repost",
        ]
        for argument in argument_list:
            if config[argument] != 0 and config[argument] != 1:
                logger.warning("%s值应为0或1,请重新输入", config[argument])
                sys.exit()

        # 验证since_date
        # since_date = config["since_date"]
        # if (not self.is_date(str(since_date))) and (not isinstance(since_date, int)):
        #     logger.warning("since_date值应为yyyy-mm-dd形式或整数,请重新输入")
        #     sys.exit()

        # 验证query_list
        query_list = config.get("query_list") or []
        if (not isinstance(query_list, list)) and (not isinstance(query_list, str)):
            logger.warning("query_list值应为list类型或字符串,请重新输入")
            sys.exit()

        # 验证write_mode
        write_mode = ["csv", "json", "mongo", "mysql", "sqlite"]
        if not isinstance(config["write_mode"], list):
            sys.exit("write_mode值应为list类型")
        for mode in config["write_mode"]:
            if mode not in write_mode:
                logger.warning(
                    "%s为无效模式，请从csv、json、mongo和mysql中挑选一个或多个作为write_mode", mode
                )
                sys.exit()
        # 验证运行模式
        if "sqlite" not in config["write_mode"] and const.MODE == "append":
            logger.warning("append模式下请将sqlite加入write_mode中")
            sys.exit()

        # 验证user_id_list
        user_id_list = config["user_id_list"]
        if (not isinstance(user_id_list, list)) and (not user_id_list.endswith(".txt")):
            logger.warning("user_id_list值应为list类型或txt文件路径")
            sys.exit()
        if not isinstance(user_id_list, list):
            if not os.path.isabs(user_id_list):
                user_id_list = (
                        os.path.split(os.path.realpath(__file__))[0] + os.sep + user_id_list
                )
            if not os.path.isfile(user_id_list):
                logger.warning("不存在%s文件", user_id_list)
                sys.exit()

        comment_max_count = config["comment_max_download_count"]


    # def is_date(self, since_date):
    #     """判断日期格式是否正确"""
    #     try:
    #         datetime.strptime(since_date, "%Y-%m-%d")
    #         return True
    #     except ValueError:
    #         return False
    def start(self):
        """运行爬虫"""
        try:
            for user_config in self.user_config_list:
                if len(user_config["query_list"]):
                    for query in user_config["query_list"]:
                        self.query = query
                        self.initialize_info(user_config)
                        js, _ = self.get_json()
                        #self.get_live_photo(js)
                else:
                    self.initialize_info(user_config)
                    js, _ = self.get_json()

                    self.get_live_photo(js)

        except Exception as e:
            logger.exception(e)
    def initialize_info(self, user_config):
        """初始化爬虫信息"""

        self.userid = user_config["user_id"]

    def get_json(self):
        proxies = {'http': 'http://127.0.0.1:10887', 'https': 'http://127.0.0.1:10887'}
        """获取网页中json数据"""

        #url = 'https://twitter.com/i/api/graphql/xrQ1bq7ADF7972q_5eyQ5Q/UserMedia?variables={"userId":"' + self.userid + '",'
        # if len(self.cursor) > 3:
        #     url = url + '"cursor":"' + self.cursor + '",'
        # url = url + '"count":20,"includePromotedContent":false,"withSuperFollowsUserFields":true,"withDownvotePerspective":false,"withReactionsMetadata":false,"withReactionsPerspective":false,"withSuperFollowsTweetFields":true,"withClientEventToken":false,"withBirdwatchNotes":false,"withVoice":true,"withV2Timeline":true}&features={"unified_cards_follow_card_query_enabled":false,"dont_mention_me_view_api_enabled":true,"interactive_text_enabled":true,"responsive_web_uc_gql_enabled":true,"vibe_api_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":false,"responsive_web_enhance_cards_enabled":false}'

        url = 'https://twitter.com/i/api/graphql/BvJ6Bf18OX5zcT31kzmmnw/UserMedia?variables={"userId":"'+ self.userid +'","count":20,"includePromotedContent":false,"withSuperFollowsUserFields":true,"withDownvotePerspective":false,"withReactionsMetadata":false,"withReactionsPerspective":false,"withSuperFollowsTweetFields":true,"withClientEventToken":false,"withBirdwatchNotes":false,"withVoice":true,"withV2Timeline":true}&features={"unified_cards_follow_card_query_enabled":false,"dont_mention_me_view_api_enabled":true,"interactive_text_enabled":true,"responsive_web_uc_gql_enabled":true,"vibe_api_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":false,"responsive_web_enhance_cards_enabled":true}'

        r = requests.get(url,  params="",headers=self.headers, proxies=proxies, verify=False)
        return r.json(), r.status_code

    def get_user_json(self):
        proxies = {'http': 'http://127.0.0.1:10887', 'https': 'http://127.0.0.1:10887'}
        """获取网页中json数据"""
        """ url = 'https://twitter.com/i/api/graphql/xrQ1bq7ADF7972q_5eyQ5Q/UserMedia?variables={"userId":"1213371022696517633","count":20,"includePromotedContent":false,"withSuperFollowsUserFields":true,"withDownvotePerspective":false,"withReactionsMetadata":false,"withReactionsPerspective":false,"withSuperFollowsTweetFields":true,"withClientEventToken":false,"withBirdwatchNotes":false,"withVoice":true,"withV2Timeline":true}&features={"dont_mention_me_view_api_enabled":true,"interactive_text_enabled":true,"responsive_web_uc_gql_enabled":true,"vibe_api_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":false,"responsive_web_enhance_cards_enabled":false}' """

        url = 'https://twitter.com/i/api/graphql/xrQ1bq7ADF7972q_5eyQ5Q/UserMedia?variables={"userId":"' + self.userid + '",'
        if len(self.cursor) > 3:
            url = url + '"cursor":"' + self.cursor + '",'
        url = url + '"count":20,"includePromotedContent":false,"withSuperFollowsUserFields":true,"withDownvotePerspective":false,"withReactionsMetadata":false,"withReactionsPerspective":false,"withSuperFollowsTweetFields":true,"withClientEventToken":false,"withBirdwatchNotes":false,"withVoice":true,"withV2Timeline":true}&features={"dont_mention_me_view_api_enabled":true,"interactive_text_enabled":true,"responsive_web_uc_gql_enabled":true,"vibe_api_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":false,"responsive_web_enhance_cards_enabled":false}'

        r = requests.get(url, params="", headers=self.headers, proxies=proxies, verify=False)
        return r.json(), r.status_code

    def get_live_photo(self, js):
        try:
            jsString = str(js)
            url_list = re.findall(r"'media_url_https': '(.*?)'", jsString)
            value_list = re.findall(r"'value': '(.*?)'", jsString)
            cursorType_list = re.findall(r"'cursorType': '(.*?)'", jsString)
            """ 图片地址"""
            for itme in url_list:
                print(itme)
                #self.download_one_file(itme, self.get_filepath("img"), "" , "")

            """ 下一页的code """
            for itme in value_list:
                print(itme)
            if len(cursorType_list) > 1 and len(url_list) > 0:
                self.cursor = value_list[len(value_list)-1]
                js, _ = self.get_json()
                self.get_live_photo(js)

        except Exception as e:
            print("数据解析失败！")
            logger.exception(e)

    def download_one_file(self, url, file_path, type, userid):
        """下载单个文件(图片/视频)"""
        try:
            url = url[:len(url)-4] + ".png"
            proxies = {'http': 'http://127.0.0.1:10887', 'https': 'http://127.0.0.1:10887'}
            file_exist = os.path.isfile(file_path)
            s = requests.Session()
            s.mount(url, HTTPAdapter(max_retries=5))
            flag = True
            try_count = 0
            start = url.split("/")
            name = start[len(start)-1]
            file_down_path = file_path + os.sep + name
            while flag and try_count < 5:
                flag = False
                downloaded = s.get(
                    url, headers=self.down_headers, proxies=proxies,  verify=False
                )
                try_count += 1
                if (
                        url.endswith(("jpg", "jpeg"))
                        and not downloaded.content.endswith(b"\xff\xd9")
                ) or (
                        url.endswith("png")
                        and not downloaded.content.endswith(b"\xaeB`\x82")
                ):
                    flag = True

            # 需要分别判断是否需要下载
            if not file_exist:
                with open(file_down_path, "wb") as f:
                    f.write(downloaded.content)

        except Exception as e:

            logger.exception(e)
    def get_filepath(self, type):
        """获取结果文件路径"""
        try:

            file_dir = (
                os.path.split(os.path.realpath(__file__))[0]
                + os.sep
                + "twitter"
                + os.sep
                + self.userid
            )
            if type == "img" or type == "video":
                file_dir = file_dir + os.sep + type
            if not os.path.isdir(file_dir):
                os.makedirs(file_dir)
            if type == "img" or type == "video":
                return file_dir
            file_path = file_dir + os.sep + self.userid + "." + type
            return file_path
        except Exception as e:
            logger.exception(e)

def get_config():
    """获取config.json文件信息"""
    config_path = os.path.split(os.path.realpath(__file__))[0] + os.sep + "config.json"
    if not os.path.isfile(config_path):
        logger.warning(
            "当前路径：%s 不存在配置文件config.json",
            (os.path.split(os.path.realpath(__file__))[0] + os.sep),
        )
        sys.exit()
    try:
        with open(config_path, encoding="utf-8") as f:
            config = json.loads(f.read())
            return config
    except ValueError:
        logger.error(
            "config.json 格式不正确，请参考 " "https://github.com/dataabc/weibo-crawler#3程序设置"
        )
        sys.exit()

def main():
    try:
        config = get_config()
        tw = Twitter(config)
        #tw.start()  # 爬取微博信息

        print((datetime.datetime.now()+datetime.timedelta(days=-1)).strftime("%Y-%m-%d %H:%M:%S"))
        if const.NOTIFY["NOTIFY"]:
            push_deer("更新了一次微博")
    except Exception as e:
        if const.NOTIFY["NOTIFY"]:
            push_deer("weibo-crawler运行出错，错误为{}".format(e))
        logger.exception(e)


if __name__ == "__main__":
    main()
